# 关键设置 - 告诉 Scrapy 在哪里找爬虫
SPIDER_MODULES = ['twitter_graphql_crawler.spiders']
NEWSPIDER_MODULE = 'twitter_graphql_crawler.spiders'

BOT_NAME = 'twitter_graphql_crawler'
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'

# 反爬配置
DOWNLOAD_DELAY = 3.5
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 5.0
AUTOTHROTTLE_MAX_DELAY = 60.0
CONCURRENT_REQUESTS_PER_DOMAIN = 1
RETRY_TIMES = 5
RETRY_HTTP_CODES = [429, 500, 502, 503, 504]

# 中间件激活
DOWNLOADER_MIDDLEWARES = {
    'twitter_graphql_crawler.middlewares.ProxyMiddleware': 543,
    'twitter_graphql_crawler.middlewares.HeadersMiddleware': 544,
    'twitter_graphql_crawler.middlewares.CustomRetryMiddleware': 550,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,  # 禁用默认重试
}

# Item Pipeline
ITEM_PIPELINES = {
    'twitter_graphql_crawler.pipelines.JsonWriterPipeline': 300,
}

# 调试配置
LOG_LEVEL = 'INFO'
COOKIES_DEBUG = True
COOKIES_ENABLED = True